package com.xiaotu.spider.project.pageprocessor.list;

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang.StringUtils;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.google.gson.Gson;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 IQYCommentListPageProcessor
 * @日期 2017年8月28日
 * @作者 王艳龙
 * @功能 爱奇艺爬取类--TV 短评
 */
@Component("IQYCommentListPageProcessor")
@Scope("prototype")
public class IQYCommentListPageProcessor extends AUrlPageProcessor
{
	
    private String fileName; //读取tvName文件路径
    private String IQYComment; //爱奇艺短评
    
    @Override
    protected String[] getStartRequestUrls()
    {
        List<JSONObject> targetList = readFileColumn(fileName); //读取excel电影名
        List<String> linkList = new ArrayList<String>();
        for (JSONObject target : targetList)
        {
            linkList.add("http://so.iqiyi.com/so/q_"
				+ KeywordEncode.encode(target.getString("tvName"))
				+ "?source=input&sr=246776524452");
        }
        String[] urlArr = new String[linkList.size()];
        linkList.toArray(urlArr);
        return urlArr;

    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        if (map.get("fileName") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        fileName = map.get("fileName") + StringUtils.EMPTY;

        if (map.get("IQYComment") == null)
            throw new SpiderException(
                    SpiderExceptionCode.PageProcessorParameterErrorException);
        IQYComment = map.get("IQYComment") + StringUtils.EMPTY;
    }

    @Override
    public String getPageDataType(Page page)
    {
    	String url = page.getUrl().toString();
		if (url.contains("http://so.iqiyi.com/so/q_"))
			return Constants.PageDataType.SEARCH_LIST;
		else if (url.contains(".html"))
			return Constants.PageDataType.TV_RECORD;
		//else if (url.contains("http://api.t.iqiyi.com/qx_api/comment/get_video_comments"))
		else if (url.contains("http://api-t.iqiyi.com/feed/get_feeds"))
			return Constants.PageDataType.TV_COMMENT;
		return Constants.PageDataType.UNKNOW;
    }

    @Override
    public void process(Page page)
    {
    	Map<String, Object> tvInfoMap = page.getRequest().getExtras();
    	String url = page.getUrl().toString();
    	if(StringUtils.isNotEmpty(url) && url.contains("so/q_")){
    		String tvName =url.substring(url.indexOf("q_")+2, url.indexOf("?source"));
    		tvInfoMap.put("tvName", tvName);
    	}
		String dataType = this.getPageDataType(page);
		switch (dataType)
		{
			case Constants.PageDataType.SEARCH_LIST:
				this.searchListProcess(page, tvInfoMap);
				break;
			case Constants.PageDataType.TV_RECORD:
				this.tvRecordProcess(page, tvInfoMap);
				break;
			case Constants.PageDataType.TV_COMMENT:
				this.tvCommentProcess(page, tvInfoMap);
				break;
			default:
				break;
		}
    }

    private void searchListProcess(Page page,
			final Map<String, Object> tvInfoMap)
	{
			// 搜索頁面queryPlayFormat
			/*String linkUrl = page.getHtml()
					.xpath("//div[@class='bottom_left']//a/@href")
					.toString();*/
    		String linkUrl = null;
			List<String> linkList =page.getHtml().xpath("//div[@class='info_item mt15']").all();
			for (int i=0;i<=linkList.size();i++) {
				String linkUrls =linkList.get(0);
				Html htmlstr = new Html(linkUrls);
				List<String> linkLists = htmlstr.xpath("//ul[@class='result_album clearfix']//li").all();
				for (int j = 0; j < linkLists.size(); j++) {
					String linkUrl2 =linkLists.get(0);
					Html htmlstr2 = new Html(linkUrl2);
					linkUrl = htmlstr2.xpath("//li[@class='album_item']//a/@href").toString();
				}
			}
			PageUtils.addTargetRequest(page,linkUrl,page.getRequest().getExtras());// 添加评论链接
			
	}
	
	private void tvRecordProcess(Page page, Map<String, Object> tvInfoMap)
	{
		/*String aid =page.getHtml().xpath("//ul[@class='play_recom_title']//li[@data-footer-recommend='recommend']/@data-aid").get();
		String qitanId =page.getHtml().xpath("//div[@id='qitancommonarea']/@data-qitancomment-qitanid").get();
		if(StringUtils.isNotEmpty(aid) && StringUtils.isNotEmpty(qitanId)){
			//获取总页数
   			String jsonStr = HttpUtils.httpGetJSONP("http://api.t.iqiyi.com/qx_api/comment/get_video_comments?aid="
					+ aid
					+ "&categoryid=2&cb=fnsucc&escape=true&need_reply=true&need_subject=true&need_total=1&page=1&page_size=20&page_size_reply=3&qitan_comment_type=1&qitancallback=fnsucc&qitanid="
					+ qitanId
					+ "&sort=add_time&t=0.0036409597378224134&tvid=0");
   			System.out.println("http://api.t.iqiyi.com/qx_api/comment/get_video_comments?aid="
					+ aid
					+ "&categoryid=2&cb=fnsucc&escape=true&need_reply=true&need_subject=true&need_total=1&page=1&page_size=20&page_size_reply=3&qitan_comment_type=1&qitancallback=fnsucc&qitanid="
					+ qitanId
					+ "&sort=add_time&t=0.0036409597378224134&tvid=0");
   			jsonStr = jsonStr.substring(jsonStr.indexOf("{"));
   			jsonStr = jsonStr.substring(0, jsonStr.lastIndexOf("}") + 1);
   			JSONObject C_json = JSONObject.fromObject(jsonStr);
   			String totalPage = C_json.getJSONObject("data").getString("count");
   			int pageSize =Integer.valueOf(totalPage);
   			//pageSize = (int) Math.ceil((int) Math.ceil(((double)pageSize)/20));
   				
   			for (int i = 1; i <= pageSize; i++)
   			{
   			System.out.println("http://api.t.iqiyi.com/qx_api/comment/get_video_comments?aid="
							+ aid
							+ "&categoryid=2&cb=fnsucc&escape=true&need_reply=true&need_subject=true&need_total=1&page="+i+"&page_size=20&page_size_reply=3&qitan_comment_type=1&qitancallback=fnsucc&qitanid="
							+ qitanId
							+ "&sort=add_time&t=0.0036409597378224134&tvid=0");
			//短评地址
			PageUtils.addTargetRequest(page,
					"http://api.t.iqiyi.com/qx_api/comment/get_video_comments?aid="
							+ aid
							+ "&categoryid=2&cb=fnsucc&escape=true&need_reply=true&need_subject=true&need_total=1&page="+i+"&page_size=20&page_size_reply=3&qitan_comment_type=1&qitancallback=fnsucc&qitanid="
							+ qitanId
							+ "&sort=add_time&t=0.0036409597378224134&tvid=0",
					tvInfoMap);
   			}
		}*/
		Html html = page.getHtml();
		String albumId = getSrings(html.toString(), Pattern.quote("albumId:") +
                "(.*?)" + Pattern.quote("tvId:"));
		String tvId = getSrings(html.toString(), Pattern.quote("tvId:") +
                "(.*?)" + Pattern.quote("cid:"));
		String wallId = getSrings(html.toString(), Pattern.quote("wallId:\'") +
                "(.*?)" + Pattern.quote("qipuId:"));
		if(StringUtils.isNotEmpty(albumId) && StringUtils.isNotEmpty(tvId) && StringUtils.isNotEmpty(wallId)){
			albumId =albumId.replace(",", "").trim();
			tvId =tvId.replace(",", "").trim();
			wallId =wallId.replace("',", "").trim();
			tvInfoMap.put("albumId", albumId);
			tvInfoMap.put("tvId", tvId);
			tvInfoMap.put("wallId", wallId);
			//短评地址
			PageUtils.addTargetRequest(page,
					"http://api-t.iqiyi.com/feed/get_feeds?callback=jQuery05221358929488265_1505367748334&authcookie="
    				+ "&device_id=pc_web&m_device_id=0d1dce3b45b75aec3efd7ba073422e8d&agenttype=118"
   					+ "&wallId="+wallId+"&feedTypes=1%2C7&count=20&top=1&hasRecomFeed=1"
   					+ "&feedId=&needTotal=1&version=1&qypid=01010011010000000000&isVideoPage=true"
   					+ "&tvid="+tvId+"&categoryid=2"
   					+ "&albumid="+albumId+"&upOrDown=0&snsTime=1&_=1505265086364",
					tvInfoMap);
			System.out.println("http://api-t.iqiyi.com/feed/get_feeds?callback=jQuery05221358929488265_1505367748334&authcookie="
	        				+ "&device_id=pc_web&m_device_id=0d1dce3b45b75aec3efd7ba073422e8d&agenttype=118"
   					+ "&wallId="+wallId+"&feedTypes=1%2C7&count=20&top=1&hasRecomFeed=1"
   					+ "&feedId=&needTotal=1&version=1&qypid=01010011010000000000&isVideoPage=true"
   					+ "&tvid="+tvId+"&categoryid=2"
   					+ "&albumid="+albumId+"&upOrDown=0&snsTime=1&_=1505265086364");
		}
	}
	
	private void tvCommentProcess(Page page, Map<String, Object> tvInfoMap)
	{
		String commentListJsonStr = page.getRawText();
		if (StringUtils.isEmpty(commentListJsonStr)
				|| !commentListJsonStr.contains("{"))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		commentListJsonStr = commentListJsonStr
				.substring(commentListJsonStr.indexOf("try{")+4,commentListJsonStr.indexOf(")}catch(e){}"));
		commentListJsonStr = commentListJsonStr
				.substring(commentListJsonStr.indexOf("{"));
		System.out.println(commentListJsonStr);
		JSONObject commentListObject = JSONObject
				.fromObject(commentListJsonStr);
		if (commentListObject == null || commentListObject.isEmpty()
				|| !commentListObject.has("data"))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		
		JSONObject commentObject = commentListObject.getJSONObject("data");
		
		if (commentObject == null || commentObject.isEmpty())
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		
		JSONArray commentList = commentObject.getJSONArray("feeds");
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
		System.out.println(commentList.size());
		for (int i = 0; i < commentList.size(); i++){
			JSONObject comment = commentList.getJSONObject(i);
			Date data = new Date(Long.valueOf(comment.getString("releaseDate")) * 1000);
			String pubtime =sdf.format(data);
			String line = StringUtils.join(new String[]{StringUtil.nullToString(page.getRequest().getExtra("type")),
					KeywordEncode.decode((String)tvInfoMap.get("tvName")), comment.getString("name"), 
					pubtime, comment.getString("description")}, SepratorUtil
	                .SEP_PIPE);
	        System.out.println(line);
	        FileUtil.writeFile(IQYComment, line + SepratorUtil.LINE_SEPRATOR);
	        if(i == commentList.size()-1){
	        	String commentId =comment.getString("commentId");
	        	String snsTime = comment.getString("snsTime");
	        	String wallId =(String)tvInfoMap.get("wallId");
	        	String albumId =(String)tvInfoMap.get("albumId");
	        	String tvId =(String)tvInfoMap.get("tvId");
	        	if(StringUtils.isNotEmpty(commentId) && commentId !=null){
	        		String linkUrl ="http://api-t.iqiyi.com/feed/get_feeds?callback=jQuery05221358929488265_1505367748334&authcookie="
	        				+ "&device_id=pc_web&m_device_id=0d1dce3b45b75aec3efd7ba073422e8d&agenttype=118"
	       					+ "&wallId="+wallId+"&feedTypes=1%2C7&count=20&top=1&hasRecomFeed=1"
	       					+ "&feedId="+commentId+"&needTotal=1&version=1&qypid=01010011010000000000&isVideoPage=true"
	       					+ "&tvid="+tvId+"&categoryid=2"
	       					+ "&albumid="+albumId+"&upOrDown=1&snsTime="+snsTime+"&_=1505265086364";
	        		System.out.println(linkUrl);
	        		PageUtils.addTargetRequest(page,linkUrl,page.getRequest().getExtras());// 添加评论链接
	        	}else{
	        		return;
	        	}
	        }
		}
	}
    /**
     * 读取文件指定列
     *
     * @param fileName 文件名称
     * @return 行列表
     */
    public static List<JSONObject> readFileColumn(String fileName)
    {
        Map<String, Object> map = new HashMap<String, Object>();
        Workbook wb = null;
        String json;
        List<JSONObject> list = new ArrayList<JSONObject>();
        try
        {
            InputStream inputStream = new FileInputStream(fileName);
            if (fileName.endsWith("xls"))
            {
                wb = new HSSFWorkbook(inputStream); // 解析xls格式
            } else if (fileName.endsWith("xlsx"))
            {
                wb = new XSSFWorkbook(inputStream); // 解析xlsx格式
            }
            Sheet sheet = wb.getSheetAt(0); // 第一个工作表

            int firstRowNum = 1;
            int lastRowNum = sheet.getLastRowNum();

            Row row = null;
            Cell cell_a1 = null;
            for (int i = firstRowNum; i <= lastRowNum; i++)
            {
                row = sheet.getRow(i);          //取得第i行
                cell_a1 = row.getCell(0);      //取得i行的第一列
                if(row.getCell(0)!=null){
                	cell_a1.setCellType(Cell.CELL_TYPE_STRING);
                    String cellValue1 = cell_a1.getStringCellValue();
                    map.put("tvName", cellValue1);
               }
                json = new Gson().toJson(map);
                JSONObject jsStr = JSONObject.fromObject(json);
                list.add(jsStr);
            }

        }
        catch (FileNotFoundException e)
        {
            e.printStackTrace();
        }
        catch (IOException e)
        {
            e.printStackTrace();
        }
        return list;
    }

    @Override
    protected String getSpiderType()
    {
        return "IQYCommentList";
    }
    
    /**
     * 获取多个目标字符串
     */
    public static String getSrings(String input, String regex)
    {
    	String result=null;
        Pattern pat = Pattern.compile(regex, Pattern.DOTALL);
        Matcher mat = pat.matcher(input);
        while (mat.find())
        {
            result = mat.group(1);
        }
        return result;
    }
}
